{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "031b3e7d-9732-487b-a5f2-ad747503c8e1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# matpltolib, seaborn\n",
    "# matpltolib是比较底层\n",
    "# seaborn是基于matpltolib\n",
    "\n",
    "# https://matplotlib.org/stable/tutorials/index.html\n",
    "# https://seaborn.pydata.org/examples/index.html\n",
    "import seaborn as sns\n",
    "\n",
    "%pylab inline\n",
    "\n",
    "# 这两行代码解决 plt 中文显示的问题\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "from IPython.display import set_matplotlib_formats\n",
    "set_matplotlib_formats('svg') # 展示的图片格式\n",
    "\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8940a478-d546-44e3-8eb7-98fddc3157c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('./糖尿病遗传风险预测挑战赛公开数据/比赛训练集.csv', encoding='gbk')\n",
    "test_df = pd.read_csv('./糖尿病遗传风险预测挑战赛公开数据/比赛测试集.csv', encoding='gbk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a7e73bd-b4a2-4aca-acab-12e9b343269f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df.head()\n",
    "\n",
    "# 病人的信息\n",
    "\n",
    "# 相同性别下 体重指数的均值~\n",
    "\n",
    "# 特征要与原始样本个数相同！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fbb46e0-9238-45e4-9084-b7702a4d0ff6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df.groupby('性别')['体重指数'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4390217e-f6d7-492b-b09b-db7aebe8f14e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 相同性别下 体重指数与均值的差异~\n",
    "\n",
    "train_df['体重指数'] - train_df['性别'].map(\n",
    "    train_df.groupby('性别')['体重指数'].mean()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "797681f1-8cea-4026-a2d1-d6196a0c5979",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['性别'].map(\n",
    "    train_df.groupby('性别')['体重指数'].mean()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c359c706-da47-4b7f-b630-30f198c079a7",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Pandas数据分组和聚合\n",
    "\n",
    "- 行索引、列索引、条件索引\n",
    "- 分组聚合"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b899593d-665f-4400-831b-1839dcb23993",
   "metadata": {},
   "source": [
    "## groupby"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14b93f06-4133-41f8-9dc8-460a2f65a82a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"https://cdn.coggle.club/Pokemon.csv\")\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ef2994a-f6fe-4390-bf60-0e3bab3f2622",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 迭代打印所有的分组数据\n",
    "for tmp_df in df.groupby('Type 1'):\n",
    "    # key， 表格\n",
    "    print(tmp_df[0], tmp_df[1].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12cbb0fd-cca7-4a87-b8c3-f541c24f00f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据Type 1分组统计HP的最大值\n",
    "df.groupby(\"Type 1\")[\"HP\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0769a7dd-cfc8-4850-bc7f-3cd6a223aac7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据Type 1分组统计Legendary的取值个数\n",
    "df.groupby(\"Type 1\")[\"Legendary\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fbadbde-5499-4914-851c-5c2f3bd4e8eb",
   "metadata": {},
   "source": [
    "## apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5da9d29f-9a1a-42ac-bd2c-44222bb734dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据Type 1分组统计HP的最大值\n",
    "df.groupby(\"Type 1\")[\"HP\"].apply(np.max)\n",
    "\n",
    "# df.groupby(\"Type 1\")[\"HP\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "175639b8-8dbc-4de1-a7e1-b87b98b55df8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据Type 1分组统计HP的最大值\n",
    "df.groupby(\"Type 1\")[\"HP\"].apply(lambda x: np.max(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d5e8146-f384-4087-9953-651afa68c63d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据Type 1分组统计HP的中位数\n",
    "df.groupby(\"Type 1\")[\"HP\"].apply(lambda x: np.percentile(x, 0.5))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1692959-4dfc-4e4a-b421-bcb2533fdd38",
   "metadata": {},
   "source": [
    "## agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d01aa21-77a2-4dd6-9847-42a3d03a79ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(\"Type 1\")[\"HP\"].agg(['mean', 'min', 'max']) \n",
    "\n",
    "# df.groupby(\"Type 1\")[\"HP\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45a57887-1af4-4956-b2a4-6fedc89ec66b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.groupby(\"Type 1\")[\"HP\"].agg(['mean', 'min', 'max']) \n",
    "\n",
    "df.groupby(\"Type 1\").agg({\n",
    "    'HP': ['mean', 'max', 'min'],\n",
    "    'Attack': ['mean', 'max', 'min'],\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c7ba5eb-970f-462a-adb8-7bd949b01335",
   "metadata": {},
   "source": [
    "## transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7c32c1-9a5c-4985-a1a8-fcd71e64bd6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()\n",
    "\n",
    "# 原始数据的每一行：一个宠物\n",
    "# Type 1分组下面 HP 最大值 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb93d9b4-dcc0-4b79-8582-a9ca562f0bbc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"Type 1\"].map(\n",
    "    {'Grass': 1, 'Fire': 2}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6663e232-4637-448d-8b01-21fbf9224687",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(\"Type 1\")[\"HP\"].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1db33abe-f789-4d67-afaa-ac3584c20926",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Type 1\"].map(\n",
    "    df.groupby(\"Type 1\")[\"HP\"].max() # series 【index】\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38a02c05-31bd-44c8-b0ea-61962c8e9db7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(\"Type 1\")[\"HP\"].transform('max') # 返回的就是和样本个数一样多的一个特征！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b604d8d-7ede-45da-acfc-2de51e5aa2d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(\"Type 1\")[\"HP\"].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b57fa0b7-1559-4eec-81fe-7c8209920ead",
   "metadata": {},
   "source": [
    "# 可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "357f1893-8bb4-432a-883a-4c1e3a0b99c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['性别'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de7beef7-98ab-439c-bd33-421a7aa5b19e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df['性别'].value_counts().plot(kind='bar')\n",
    "# plot pandas里面的函数，对series进行可视化\n",
    "# kind 表示画图方法\n",
    "# plot 底层也是matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "037a78e4-b0fe-478a-8e40-e6ac0bb4c5e5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df['性别'].value_counts().plot(kind='barh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8927b315-2588-426d-83e2-ac27ed64a189",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df.groupby(['患有糖尿病标识'])['性别'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7344d698-9ef3-4a10-87fa-36cbfe573001",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.countplot(x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffd321dd-fc0f-4584-a96f-a601f3be0aa9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.boxplot(y='出生年份', x='患有糖尿病标识', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d32cc790-0652-49e8-a3dd-bafd5b50db6c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.boxplot(y='出生年份', x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "657bce87-6bdf-4d4a-a901-4d233a5d6c2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.boxplot(y='体重指数', x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c70c1f7-6f21-451e-88f6-990c0e6f19d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(y='体重指数', x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68945e0f-97c5-4eec-82a8-6b3de0388b78",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.kdeplot(x='体重指数', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3174d9e4-7428-4bc5-87eb-928f0e12b28c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(y='舒张压', x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7862a453-8ff2-4330-a5de-b2f333184771",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.scatterplot(y='舒张压', x='体重指数', hue='患有糖尿病标识',data=train_df)\n",
    "\n",
    "# 不同的画图方法 X Y 具体的类型是不同的！\n",
    "# 【参与画图的数据类型 选择 合适的图形】"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f58ca76-583c-4c78-926e-ab2f3fa9e75c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.kdeplot(y='舒张压', x='体重指数', hue='患有糖尿病标识',data=train_df, fill=True, alpha=0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "125e8e63-311e-47d1-8630-4c28dd0e1093",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(y='口服耐糖量测试', x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "500a291b-95f7-4262-9a44-c1a4a8af4d77",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.scatterplot(y='口服耐糖量测试', x='体重指数', hue='患有糖尿病标识',data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb30c20e-a8cd-4ca2-a8ed-01380844c5e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.scatterplot(y='口服耐糖量测试', x='胰岛素释放实验', hue='患有糖尿病标识',data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02cce6bc-ac83-4463-9e97-347f98e4cfd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.scatterplot(y='肱三头肌皮褶厚度', x='胰岛素释放实验', hue='患有糖尿病标识',data=train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8b5315e-d950-4ad6-97d8-112b086a766f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(y='肱三头肌皮褶厚度', x='患有糖尿病标识', hue='性别', data=train_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbfa66fe-80fa-4f93-9d09-65637d303def",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06942635-6303-4852-a935-c075fc6d2c3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame({\n",
    "    'student_id': [1,2,3,4,5,6,7],\n",
    "    'country': ['China', 'USA', 'UK', 'Japan', 'Korea', 'China', 'USA'],\n",
    "    'education': ['Master', 'Bachelor', 'Bachelor', 'Master', 'PHD', 'PHD', 'Bachelor'],\n",
    "    'target': [1, 0, 1, 0, 1, 0, 1]\n",
    "})\n",
    "df.head(10)\n",
    "df.style.set_table_attributes('style=\"font-size: 17px\"')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bff8224d-4eae-403c-a150-99a15316e83b",
   "metadata": {},
   "source": [
    "### One Hot Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4613e4ba-caa4-4474-a1b1-c7b96be777e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.get_dummies(df, columns=['education'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c7dfa4b-054d-43aa-bc88-2264f9c8a8dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "ohe = OneHotEncoder()\n",
    "ohe.fit_transform(df[['country']]).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5988398-31de-4b37-bd5e-7932ca23a579",
   "metadata": {},
   "source": [
    "### Label Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd2f5a56-cb26-43cf-8a2e-1c4aeea7f6fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "df['country_LabelEncoder'] = le.fit_transform(df['country'])\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3a257f1-2ebf-46fb-870f-63598e035bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['country_LabelEncoder'] = pd.factorize(df['country'])[0]\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b767522d-49e7-4710-b878-314a15710418",
   "metadata": {},
   "source": [
    "### Ordinal Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "303d9ea7-eee0-4add-be16-85394fe72d51",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['education'] = df['education'].map(\n",
    "                    {'Bachelor': 1, \n",
    "                    'Master': 2, \n",
    "                    'PHD': 3})\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27ead68f-198a-42de-ad29-ffbdfef4d32d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(['Bachelor','Master','PHD', 'Postdoctoral']).map(\n",
    "                    {'Bachelor': 1, \n",
    "                    'Master': 2, \n",
    "                    'PHD': 3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0c579db-6ff8-4f90-95c7-4386a5237153",
   "metadata": {},
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "encoder = ce.SumEncoder(cols= ['country'])\n",
    "\n",
    "pd.concat([df, encoder.fit_transform(df['country'], df['target']).iloc[:, 1:]], axis=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
